# coding=utf-8
import time

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
# from urllib.request import quote, unquote
from lxml import html

class QhinfoSpider(object):
    name = 'qhinfo'
    allowed_domains = ['www.fmz.com']
    start_urls = ['https://www.fmz.com/']

    def close(self):
        self.browser.close()

    def start_requests(self):
        print("init---------------")
        opt = Options()
        opt.add_argument('--no-sandbox')
        opt.add_argument('--disable-gpu')
        opt.add_experimental_option('excludeSwitches', ['enable-automation'])
        opt.add_argument("--disable-dev-usage")
        desired_capabilities = DesiredCapabilities.CHROME  # 修改页面加载策略
        desired_capabilities["pageLoadStrategy"] = "none"
        opt.add_argument("–disable-gpu")
        opt.add_argument("log-level=3")
        opt.add_experimental_option('excludeSwitches', ['enable-logging'])
        opt.add_argument('blink-settings=imagesEnabled=false')
        opt.add_argument('disable-infobars')
        # opt.add_argument('--headless')
        self.chrome_driver = 'chromedriver.exe'
        self.browser = webdriver.Chrome(self.chrome_driver, options=opt)
        self.browser.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
            "source": """
            Object.defineProperty(navigator, 'webdriver', {
            get: () => undefined
            })
        """
        })
        self.browser.set_window_size(1920, 900)
        self.browser.set_page_load_timeout(30)
        # url = "https://sh.lianjia.com/xiaoqu/5011000014141/"
        # self.parse(url)
        self.browser.get("https://sh.lianjia.com/xiaoqu/?from=rec")
        self.browser.set_page_load_timeout(10000)
        while True:
            response = html.fromstring(self.browser.page_source)
            getalllist = response.xpath(".//ul[@class='listContent']/li")
            for itlist in getalllist:
                iturl = itlist.xpath(".//div[@class='info']/div[@class='title']/a/@href")[0]
                self.parse(iturl)
            if response.xpath("/html/body/div[4]/div[1]/div[3]/div[2]/div/a[last()]/text()")[0]=="下一页":
                self.browser.find_element_by_xpath("/html/body/div[4]/div[1]/div[3]/div[2]/div/a[last()]").click()
            else:
                break

    def parse(self,url):
        # infoCode = input("请输入您的编号：")



        self.browser.get(url)
        self.browser.set_page_load_timeout(10000)
        time.sleep(2)
        # self.browser.execute_cdp_cmd('var q=document.documentElement.scrollTop=2500')
        self.browser.execute_script("var q=document.documentElement.scrollTop=2500")
        response = html.fromstring(self.browser.page_source)
        item = {}
        item["title"]  = self.parserTie(response)
        item["price"] = response.xpath(".//span[@class='xiaoquUnitPrice']/text()")[0] + "元/平方米"
        item["kaifa"] = response.xpath(".//div[@class='xiaoquInfo']/div[5]/span[2]/text()")[0]
        item["inner"] = response.xpath(".//div[@class='xiaoquInfo']/div[8]/span[2]/text()")[0].replace("/","")

        allli = self.browser.find_elements_by_xpath(".//ul[@class='aroundType']/li")
        for alliitem in allli:
            alliitem.click()
            time.sleep(0.8)
            if allli.index(alliitem) == 0:
                try:
                    item["ditie"] = self.browser.find_element_by_xpath(".//ul[@class='itemBox']/li[1]/div/div[1]/span[2]").get_attribute("innerHTML") + self.browser.find_element_by_xpath(".//ul[@class='itemBox']/li[1]/div/div[1]/span[4]").get_attribute("innerHTML")
                except Exception:
                    item["ditie"] = "暂无"
            if allli.index(alliitem) == 1:
                allSuper11 = self.browser.find_elements_by_xpath(".//div[@class='itemTagBox']/div")
                for allSuperItem in allSuper11:

                    if allSuper11.index(allSuperItem)  == 0:
                        try:
                            item["youeryuan"] = self.browser.find_element_by_xpath(".//ul[@class='itemBox']/li[1]/div/div[1]/span[2]").get_attribute("innerHTML") + self.browser.find_element_by_xpath(".//ul[@class='itemBox']/li[1]/div/div[1]/span[4]").get_attribute("innerHTML")
                        except Exception:
                            item["youeryuan"] = "暂无"
                    if allSuper11.index(allSuperItem)  == 1:
                        allSuperItem.click()
                        time.sleep(1.2)
                        try:
                            item["xiaoxue"] = self.browser.find_element_by_xpath(".//ul[@class='itemBox']/li[1]/div/div[1]/span[2]").get_attribute("innerHTML") + self.browser.find_element_by_xpath(".//ul[@class='itemBox']/li[1]/div/div[1]/span[4]").get_attribute("innerHTML")
                        except Exception:
                            item["xiaoxue"] = "暂无"
                    if allSuper11.index(allSuperItem)  == 2:
                        allSuperItem.click()
                        time.sleep(1.2)
                        try:
                            item["zhongyue"] = self.browser.find_element_by_xpath(".//ul[@class='itemBox']/li[1]/div/div[1]/span[2]").get_attribute("innerHTML") + self.browser.find_element_by_xpath(".//ul[@class='itemBox']/li[1]/div/div[1]/span[4]").get_attribute("innerHTML")
                        except Exception:
                            item["zhongyue"] = "暂无"
                    if allSuper11.index(allSuperItem)  == 3:
                        allSuperItem.click()
                        time.sleep(1.2)
                        try:
                            item["daxue"] = self.browser.find_element_by_xpath(".//ul[@class='itemBox']/li[1]/div/div[1]/span[2]").get_attribute("innerHTML") + self.browser.find_element_by_xpath(".//ul[@class='itemBox']/li[1]/div/div[1]/span[4]").get_attribute("innerHTML")
                        except Exception:
                            item["daxue"] = "暂无"
            if allli.index(alliitem) == 2:
                try:
                    item["yiliao"] = self.browser.find_element_by_xpath(".//ul[@class='itemBox']/li[1]/div/div[1]/span[2]").get_attribute("innerHTML") + self.browser.find_element_by_xpath(".//ul[@class='itemBox']/li[1]/div/div[1]/span[4]").get_attribute("innerHTML")
                except Exception:
                    item["yiliao"] = "暂无"
            if allli.index(alliitem) == 3:
                allSuper = self.browser.find_elements_by_xpath(".//div[@class='itemTagBox']/div")
                for allSuperItem in allSuper:
                    if allSuper.index(allSuperItem)  == 0:
                        try:
                            item["shanchang"] = self.browser.find_element_by_xpath(".//ul[@class='itemBox']/li[1]/div/div[1]/span[2]").get_attribute("innerHTML") + self.browser.find_element_by_xpath(".//ul[@class='itemBox']/li[1]/div/div[1]/span[4]").get_attribute("innerHTML")
                        except Exception:
                            item["shanchang"] = "暂无"
                    if allSuper.index(allSuperItem)  == 1:
                        allSuperItem.click()
                        time.sleep(1.2)
                        try:
                            item["chaoshi"] = self.browser.find_element_by_xpath(".//ul[@class='itemBox']/li[1]/div/div[1]/span[2]").get_attribute("innerHTML") + self.browser.find_element_by_xpath(".//ul[@class='itemBox']/li[1]/div/div[1]/span[4]").get_attribute("innerHTML")
                        except Exception:
                            item["chaoshi"] = "暂无"
                    if allSuper.index(allSuperItem)  == 2:
                        allSuperItem.click()
                        time.sleep(1.2)
                        try:
                            item["shichang"] = self.browser.find_element_by_xpath(".//ul[@class='itemBox']/li[1]/div/div[1]/span[2]").get_attribute("innerHTML") + self.browser.find_element_by_xpath(".//ul[@class='itemBox']/li[1]/div/div[1]/span[4]").get_attribute("innerHTML")
                        except Exception:
                            item["shichang"] = "暂无"
            if allli.index(alliitem) == 4:
                allSuper = self.browser.find_elements_by_xpath(".//div[@class='itemTagBox']/div")
                for allSuperItem in allSuper:
                    if allSuper.index(allSuperItem)  == 0:
                        try:
                            item["yinhang"] = self.browser.find_element_by_xpath(".//ul[@class='itemBox']/li[1]/div/div[1]/span[2]").get_attribute("innerHTML") + self.browser.find_element_by_xpath(".//ul[@class='itemBox']/li[1]/div/div[1]/span[4]").get_attribute("innerHTML")
                        except Exception:
                            item["yinhang"] = "暂无"
                    if allSuper.index(allSuperItem)  == 1:
                        allSuperItem.click()
                        time.sleep(1.2)
                        try:
                            item["atm"] = self.browser.find_element_by_xpath(".//ul[@class='itemBox']/li[1]/div/div[1]/span[2]").get_attribute("innerHTML") + self.browser.find_element_by_xpath(".//ul[@class='itemBox']/li[1]/div/div[1]/span[4]").get_attribute("innerHTML")
                        except Exception:
                            item["atm"] = "暂无"
                    if allSuper.index(allSuperItem)  == 2:
                        allSuperItem.click()
                        time.sleep(1.2)
                        try:
                            item["canting"] = self.browser.find_element_by_xpath(".//ul[@class='itemBox']/li[1]/div/div[1]/span[2]").get_attribute("innerHTML") + self.browser.find_element_by_xpath(".//ul[@class='itemBox']/li[1]/div/div[1]/span[4]").get_attribute("innerHTML")
                        except Exception:
                            item["canting"] = "暂无"
        print(item)
        with open("data.csv","a",encoding="utf-8") as f:
            f.write(",".join(list(item.values())))
            f.write("\n")



    def parserTie(self,response):
        title = response.xpath(".//h1[@class='detailTitle']/text()")[0]
        return title


    def parseinfo(self, response):

        print("*" * 50)
        print("\n")
            # print(key,value)
        print("\n")
        print("*" * 50)
        self.browser.close()


if __name__ == "__main__":
    obj = QhinfoSpider()
    obj.start_requests()



